import sys
import os
import signal
import threading
import subprocess
import tempfile
import re
import unittest
import shutil
import datetime
import time
import traceback
import tools_lib
import xml.dom.minidom
import fileinput
import glob
import pprint

logLevel = 10
debconfKdeFrontendPath = "/usr/share/perl5/Debconf/FrontEnd/Kde.pm"
debconfGnomeFrontendPath = "/usr/share/perl5/Debconf/FrontEnd/Gnome.pm"

def englishEnviron():
    env = os.environ
    env["LC_ALL"] = "C"
    return env

def getLanguagesNames():
    if os.environ.has_key("LANGUAGE"):
        languages = os.environ["LANGUAGE"].split(":")
        dlog(5, "Languages: %s" % repr(languages))
        return languages
    return []

def getText(nodelist):
    rc = ""
    if isinstance(nodelist, list):
        for node in nodelist:
            if node.nodeType == node.TEXT_NODE:
                rc = rc + node.data
        return rc
    if nodelist.nodeType == nodelist.TEXT_NODE:
        rc = rc + node.data
    return rc

def dlog(level, message):
    if level <=logLevel:
        print message

def safeDirectoryRemove(directoryPath):
    print "Removing directory not implemented, path=%s" % directoryPath
    #if directoryPath != "/" and directoryPath != "/tmp" and directoryPath != "/home"
    #   shutil.rmtree()

def runSimpleCommand(command):
    popenObj = subprocess.Popen(
        command,
        bufsize = 1, #0 - unbuffered, 1 - line-buffered, >1 bufsize
        stdout=subprocess.PIPE
        )
    output = popenObj.communicate()
    returncode = popenObj.wait()
    return returncode
def runSimpleCommandWithOutput(command):
    popenObj = subprocess.Popen(
        command,
        bufsize = 1, #0 - unbuffered, 1 - line-buffered, >1 bufsize
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
        )
    output = popenObj.communicate()
    returncode = popenObj.wait()
    return (returncode, output)

def runSimpleCommandWithTextOutputAndInput(command, inputText):
    popenObj = subprocess.Popen(
        command,
        bufsize = 1, #0 - unbuffered, 1 - line-buffered, >1 bufsize
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
        )
    output = popenObj.communicate(inputText)
    returncode = popenObj.wait()
    return (returncode, output[0])

def runSimpleCommandWithTextOutput(command):
    returncode, out = runSimpleCommandWithOutput(command)
    return (returncode, out[0])

#returns returncode, stdout contents, stderr contents
def runSimpleCommandWithTextOutputAndStderr(command):
    returncode, out = runSimpleCommandWithOutput(command)
    return (returncode, out[0], out[1])

#Adds newLine to file contents if it does not contain line matching regexp
#Returns new file contents
def addLineToFileContentsIfNotMatching(regexp, newLine, currentContents):
    newContents = []
    regex = re.compile(regexp)
    lineFound = False
    endOfLine = True
    for line in currentContents:
        dlog(3, "Analyzing line: %s" % line)
        if len(line) > 0 and line[-1] == "\n":
            endOfLine = True
        else:
            endOfLine = False
        matchObj = regex.match(line)
        if matchObj != None:
            dlog(3, "Line %s matches" % line)
            lineFound = True
        else:
            dlog(3, "Line %s does not match" % line)
        newContents.append(line)
    if endOfLine == False:
        newContents.append("\n")
    #now add new line for this user
    if lineFound == False:
        newContents.append(newLine)
    return newContents
    
#Filters out all lines matching given regexp
#Returns new file (contents, replaced), where replaced is True iff something was replaced in file
def filterOutFileContentsMatchingLines(regexp, currentContents):
    newContents = []
    regex = re.compile(regexp)
    lineFound = False
    for line in currentContents:
        dlog(10, "Analyzing line: %s" % line)
        if regex.match(line):
            dlog(10, "Line %s matches, skipping" % line)
            lineFound = True
        else:
            newContents.append(line)
            dlog(10, "Line %s does not match" % line)
    return (newContents, lineFound)
    
def makeFileBackup(filePath, backupsPath = None):
    maxBackups = 200
    if backupsPath == None:
        backupFilePath = filePath
    else:
        backupFilePath = backupsPath + "/" + os.path.basename(filePath)
        
    backupBaseName = backupFilePath + "." + datetime.date.today().isoformat()
        
    for i in range(1, maxBackups):
        backupName = backupBaseName + "." + str(i)
        if os.path.exists(backupName):
            continue
        if not os.path.exists(os.path.dirname(backupName)):
            os.makedirs(os.path.dirname(backupName))
        shutil.copyfile(filePath, backupName)
        return
    raise Exception, "More than %d backups, someone might be trying to do something nasty! Filename=%s" % (maxBackups, filePath)

def writeConfigFile(filePath, fileContents, backupsPath = None):
    """Writes config file, keeps backup of previous file contents.
    """
    if os.path.exists(filePath):
        makeFileBackup(filePath, backupsPath)
    f = open(filePath,"w")
    try:
        f.write(fileContents)
    finally:
        f.close()
        
def readTemplateFile(templateFileName, templateVariables = {}):
    filePath = tools_lib.thisScriptDirectory() + "/file_templates/" + templateFileName
    f = open(filePath,"r")
    try:
        contents = f.readlines()
    finally:
        f.close()
    #FIXME: replace template variables in lines
    return "".join(contents)
    

class LongActionWrapper:
    def __init__(self, lineOutputFunction, heartbeatFunction, heartbeatIntervalMs = 1000):
        self.lineOutputFunction = lineOutputFunction
        self.heartbeatFunction = heartbeatFunction
        self.heartbeatIntervalMs = heartbeatIntervalMs
    def runCommand(self, command, workingDirectory = None, env = None):
        import subprocess
        popenObj = subprocess.Popen(
            command,
            bufsize = 0, #0 - unbuffered, 1 - line-buffered, >1 bufsize
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            stderr = subprocess.STDOUT,
            cwd = workingDirectory,
            env = env
            )
        finishedOk = False
        returncode = -999
        try:
            import select
            pollObj = select.poll()
            pollObj.register(popenObj.stdout)
            dots = ""
            heartbeatCounter = 0
            line = ""
            while True:
                dlog(11, "running poll()")
                events = pollObj.poll(self.heartbeatIntervalMs) #milliseconds
                dlog(12, "poll events=%s" % repr(events))
                if len(events) > 0:
                    dlog(11, "read")
                    byte = popenObj.stdout.read(1)
                    dlog(11, "byte=%s" % byte)
                    line += byte
                    dlog(11, "before if")
                    if (len(byte) == 0 and len(line) > 0) or byte == "\n":
                        dlog(11, "ending line")
                        if self.lineOutputFunction(line.decode("utf_8"), popenObj.stdin) == False:
                            dlog(10, "Line output function %s returned false" % repr(self.lineOutputFunction))
                            returncode = -1000
                            return returncode
                        line = ""
                    if len(byte) == 0:
                        dlog(11, "end of file")
                        #eof
                        break
                    dlog(11, "nothing, continue")
                    continue
                else:
                    heartbeatCounter += 1
                    self.heartbeatFunction(heartbeatCounter)
            returncode = popenObj.wait()
            finishedOk = True
        finally:
            if finishedOk == False:
                print "Exception occurred, trying to kill process with TERM, pid=%d" % popenObj.pid
                try:
                    os.kill(popenObj.pid, signal.SIGTERM)
                except OSError:
                    pass
                time.sleep(1.0)
                print "Exception occurred, trying to kill process with KILL, pid=%d" % popenObj.pid
                try:
                    os.kill(popenObj.pid, signal.SIGKILL)
                except OSError:
                    pass
        print "Command %s finished with return code %d" % (repr(command), returncode)
        return returncode

class OneClickInstallBackend:
    def __init__(self, frontend, fileName):
        self.frontend = frontend
        self.repoFileName = fileName
        self.aptKeyringLocation = "/etc/apt/trusted.gpg"
        self.aptTrustDbLocation = "/etc/apt/trustdb.gpg"
    def startWizard(self):
        #thread1 = threading.Thread(group=None, target=threadfunc, name=None, args=(win), kwargs={})
        thread = threading.Thread(None, self.start, None, ["param1"])
        thread.start()
    def addAptLine(self, driverName):
        blacklistFilePath = "/etc/modprobe.d/blacklist"
        f = open(blacklistFilePath,"r")
        try:
            currentContents = f.readlines()
        finally:
            f.close()
        newContents = addLineToFileContentsIfNotMatching(r"^\s*blacklist\s+"+re.escape(driverName)+r"\b", "blacklist " + re.escape(driverName) + "\n", currentContents)
        writeConfigFile(blacklistFilePath, "".join(newContents), "/etc/modprobe.d.backups")
        
    def packageInstalled(self, packageName):
        #try:
        output = subprocess.Popen(
            ["dpkg-query", "-W", "-f${Status}\n", packageName], 
            stdout=subprocess.PIPE, 
            env = englishEnviron()
            ).communicate()[0]
        #except OSError, e:
            #print >>sys.stderr, "Execution failed:", e
        print "output = %s" % repr(output)
        print "output.split() = %s" % repr(output.split())
        installed = "installed" in output.split()
        return installed
    def packageAvailableInRepositories(self, packageName):
        output = subprocess.Popen(
            ["apt-cache", "madison", packageName], 
            stdout=subprocess.PIPE, 
            env = englishEnviron()
            ).communicate()[0]
        print "madison output = %s" % repr(output)
        print "madison output.split() = %s" % repr(output.split("\n"))
        lines = output.split("\n")
        for line in lines:
            words = line.split()
            if len(words) == 0:
                continue
            if words[-1] == "Packages":
                #package is in one of the repositories
                return True
        return False
    def addMainRepository(self):
        while True:
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("Adding CD-ROM with software"),
                    "nextEnabled" : True,
                    "text": _("<p>Please prepare CD-ROM which you used to install your system and click \"Next\".</p>"),
                    }
                )
            self.frontend.waitForInput()
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "actionProgress",
                    "pageTitle": _("Adding CD with software"),
                    "progressLabel" : _("Adding CD")
                    }
                )
            actionWrapper = LongActionWrapper(
                self.installPackageLineOutput,
                self.addCdHeartbeat
                )
            dlog(10, "Running apt-cdrom add")
            returncode = actionWrapper.runCommand(
                ["apt-cdrom", "add"],
                env = englishEnviron()
                )
            if returncode != 0:
                tmpText = _("Adding CD failed")
                self.frontend.performAction("progressInfo", {"status": False, "nextEnabled" : True, "text": tmpText, "details": tmpText})
                self.frontend.waitForInput()
                return False
            return True
    def makeSureSoftwareInstalled(self, packageName, fromBundledDebs = False):
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "actionProgress",
                "pageTitle": _("Installing software"),
                "progressLabel" : _("Installing software")
                }
            )
        print "Checking availability of package %s" % packageName
        self.frontend.performAction("progressInfo", {"text": _("Checking for availability of necessary software")})
        realPackageName = packageName
        if fromBundledDebs == True:
            realPackageName = self.getDebPackageName(packageName)
            if realPackageName == None:
                print "Cannot find the name of deb package %s failed" % packageName
                self.frontend.performAction("progressInfo", {"status": False, "text": _("Failed to install software")})
                return False
        
        installed = self.packageInstalled(realPackageName)
        if installed == True:
            print "Package %s is installed" % packageName
            return True
        
        if fromBundledDebs == True:
            if self.installDebPackage(packageName) == True:
                print "Bundled package %s has been installed" % packageName
                self.frontend.performAction("progressInfo", {"status": True, "text": _("Software installed successfully")})
                return True
            else:
                print "Installation of bundled package %s failed, trying to install from repository" % packageName
        
        if self.packageAvailableInRepositories(packageName) == False:
            print "Package %s is not available in repositories, proceeding to adding repository" % packageName
            if self.addMainRepository() == False:
                print "Failed to add main repository"
                return False
            print "Main repository added successfully"
        else:
            print "Package %s is available in repositories, proceeding to installation" % packageName
        if self.installPackage(packageName) == True:
            print "Package %s has been installed" % packageName
            self.frontend.performAction("progressInfo", {"status": True, "text": _("Software installed successfully")})
        else:
            print "Installation of package %s failed" % packageName
            self.frontend.performAction("progressInfo", {"status": False, "text": _("Failed to install software")})
            return False
        return True
    def installPackageLineOutput(self, line, stdin):
        dlog(10, "Entered installPackageLineOutput, line=%s" % line)
#        if line.find("Please insert a Disc in the drive and press enter") >= 0:
        if line.find("Waiting for disc") >= 0 or line.find("Please insert the disc labeled") >= 0:
            self.frontend.performAction(
                "showPopupDialog", 
                {
                    "message": _("Please insert installation CD"),
                    "caption": _("Insert CD"),
                    "button0Text" : _("OK, CD is in the drive"),
                    "button1Text" : _("Cancel inserting CD")
                    }
                )
            output = self.frontend.waitForInput()
            if output["result"] == 0:
                #OK
                dlog(3, "OK pressed, sending Enter to process")
                stdin.write("\n")
            else:
                tmpText = _("Cancelled CD insertion upon user request")
                self.frontend.performAction("progressInfo", {"status": False, "text" : tmpText, "details": tmpText})
                dlog(1, "User cancelled CD insertion")
                self.frontend.waitForInput()
                return False
        self.frontend.performAction("progressInfo", {"details": line})
        return True
    def installPackageHeartbeat(self, counter):
        self.frontend.raiseIfForceTerminateRequested()
        dots = ""
        for i in range(0, counter % 4):
            dots += "."
        self.frontend.performAction("progressInfo", {"text": _("Installing") + dots })
    def genericLineOutput(self, line, stdin):
        self.frontend.performAction("progressInfo", {"details": line})
        return True
    def genericActionHeartbeat(self, counter):
        self.frontend.raiseIfForceTerminateRequested()
        dots = ""
        for i in range(0, counter % 4):
            dots += "."
        self.frontend.performAction("progressInfo", {"text": _("Please wait") + dots })
    def addCdHeartbeat(self, counter):
        self.frontend.raiseIfTerminateRequested()
        dots = "." * (counter % 4)
        self.frontend.performAction("progressInfo", {"text": _("Please wait") + dots })
        
    def installDebPackage(self, packageName):
        if self.warnNotSigned() == False:
            return False
        
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "actionProgress",
                "pageTitle": _("Installation of software"),
                "progressLabel" : _("Installing")
                }
            )
        
        self.frontend.performAction("progressInfo", {"text": _("Installing software...")})
        
        environment = os.environ
        
        if os.path.exists(debconfKdeFrontendPath):
            dlog(1, "Using KDE debconf frontend")
            environment["DEBIAN_FRONTEND"] = "kde"
        else:
            if os.path.exists(debconfGnomeFrontendPath):
                dlog(1, "Using GNOME debconf frontend")
                environment["DEBIAN_FRONTEND"] = "gnome"
        
        actionWrapper = LongActionWrapper(
            self.installPackageLineOutput,
            self.installPackageHeartbeat
            )
        
        returncode = actionWrapper.runCommand(
            ["dpkg", "--status-fd", "1", "-i", packageName],
            env = environment
            )
        
        if returncode == 0:
            print "Package %s has been installed successfully" % packageName
            self.frontend.performAction("progressInfo", {"status": True, "text": _("Software installed successfully")})
        else:
            tmpText = _("Failed to install packages")
            self.frontend.performAction("progressInfo", {"status": False, "nextEnabled" : True, "text": tmpText, "details": tmpText})
            self.frontend.waitForInput()
            print "Installation of package %s failed with error %d" % (packageName, returncode)
            return False
        return True
    
    def installPackages(self, packageNames, aptOptions = None):
        self.frontend.performAction("progressInfo", {"text": _("Installing software...")})
        
        actionWrapper = LongActionWrapper(
            self.installPackageLineOutput,
            self.installPackageHeartbeat
            )

        environment = os.environ
        
        command = ["apt-get", "install", "--assume-yes"]

        if os.path.exists(debconfKdeFrontendPath):
            dlog(1, "Using KDE debconf frontend")
            environment["DEBIAN_FRONTEND"] = "kde"
        else:
            if os.path.exists(debconfGnomeFrontendPath):
                dlog(1, "Using GNOME debconf frontend")
                environment["DEBIAN_FRONTEND"] = "gnome"
        
        if aptOptions != None:
            command.extend(aptOptions)
        command.extend(packageNames)
        command.extend(["-o", "APT::Status-Fd=1"])
        
        returncode = actionWrapper.runCommand(
            command,
            env = environment
            )
        
        if returncode == 0:
            print "Packages %s have been installed successfully" % repr(packageNames)
        else:
            tmpText = _("Failed to install packages")
            self.frontend.performAction("progressInfo", {"status": False, "nextEnabled" : True, "text": tmpText, "details": tmpText})
            self.frontend.waitForInput()
            print "Installation of packages %s failed with error %d" % (repr(packageNames), returncode)
            return False
        return True
    def start(self, params):
        self.mainTempDir = tempfile.mkdtemp()
        try:
            try:
                self.doStart(params)
            except self.frontend.TerminationRequested:
                print "User requested termination, exiting..."
            except:
                details = _("Exception: %s\nValue: %s\nBacktrace:\n%s\n") % (sys.exc_info()[0], sys.exc_info()[1], "".join(traceback.format_tb(sys.exc_info()[2])))
                print "%s" % details.encode("utf_8")
                self.frontend.performAction("showFatalError", {"message": _("Fatal error occurred in background tasks.\nPlease send bug report to author, attaching full output from console."), "details" : details})
                raise
        finally:
            dlog(1, "final statement in backend start routine")
            safeDirectoryRemove(self.mainTempDir)
            self.frontend.backendTerminated()
        dlog(1, "exiting backend start routine")
        
    def repoFileIsSigned(self):
        dlog(10, "Checking if repo file is signed")
        f = open(self.repoFileName,"r")
        try:
            firstLine = f.readline()
        finally:
            f.close()
        dlog(10, "First line of repo file: %s" % firstLine)
        if firstLine.find("BEGIN PGP SIGNED MESSAGE") >= 0:
            return True
        return False
    
    def warnNotSigned(self):
        self.frontend.performAction(
            "setNextPage",
            {
                "pageName": "simpleChoice",
                "pageTitle" : _("File not signed"),
                "text": _("<p><b>This installation file is not signed!</b></p><p>Please proceed <b>only if you trust the entity from which you got this file</b> as software from unknown source might compromise your system security, by installing spying programs, malware or intercepting your password and other confidential information. If you do not trust the author of installation file, please choose <i>Cancel installation</i> to abort installation. </p>"),
                "choices": {
                    "cancel": {"text": _("Cancel installation"), "checked" : True },
                    "proceed": {"text": _("Proceed with installation") }
                    }
                }
            )
        output = self.frontend.waitForInput()

        if output["choice"] != "proceed":
            return False
        return True
        
    def parseAndVerifyRepoFile(self):
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "actionProgress",
                "pageTitle": _("Verifying digital signature"),
                "progressLabel" : _("Verifying digital signature")
                }
            )
        self.frontend.performAction("progressInfo", {"text": _("Verifying")})
        
        if self.repoFileIsSigned():
            if not os.path.exists(os.environ["HOME"] + "/.gnupg"):
                #run gpg once to get rid of messages about creating directories
                #first time it is run.
                dlog(1, "Dummy gpg run to get rid of warnings in real run")
                runSimpleCommandWithTextOutputAndStderr(["/usr/bin/gpg", "--keyring", self.aptKeyringLocation, "--trustdb-name" , self.aptTrustDbLocation, "--batch", "--no-tty", "-o", "-", self.repoFileName])
            
            dlog(10, "Running gpg to verify and unpack repo file")
            #"-o -" is necessary to avoid warning about unknown suffix
            #using trust-model always, as keys added by apt-key
            #must be added by root, so they are trusted.
            returncode, text, stderrmsg = runSimpleCommandWithTextOutputAndStderr(["/usr/bin/gpg", "--trust-model", "always", "--keyring", self.aptKeyringLocation, "--trustdb-name" , self.aptTrustDbLocation, "--batch", "--no-tty", "-o", "-", self.repoFileName])
            
            if returncode != 0:
                tmpText = _("Verification failed")
                self.frontend.performAction("progressInfo", {"status": False, "nextEnabled" : True, "text": tmpText, "details": _("<p>Verification of digital signature failed</p>Installation cannot continue</p>")})
                self.frontend.waitForInput()
                return False
            self.frontend.performAction("progressInfo", {"text": _("Verification successful")})
            
            #Show who signed the file and warn him he should
            #proceed only if he trusts the signer.
            dlog(2, "gpg stderr output=%s" % stderrmsg.decode('utf-8'))
            signerMessage = stderrmsg.decode('utf-8').replace("\n", "<br>\n").replace("gpg:", "")
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("Signer verification"),
                    "nextEnabled" : True,
                    "finishEnabled" : False,
                    "text": _("<p><u>The installation file signer information:</u></p><p>%s</p><p>Please proceed <b>only if you trust the entity which signed the file</b> as software from unknown source might compromise your system security. If you do not trust the signer, please click Cancel to abort installation. </p>") % signerMessage
                    }
                )
            command = self.frontend.waitForInput()
            self.repoFileText = text
        else:
            #file not signed
            if self.warnNotSigned() == False:
                return False
            
            f = open(self.repoFileName,"r")
            try:
                self.repoFileText = f.read()
            finally:
                f.close()
        
        if self.parseRepoFile() == False:
            return False
        return True
    def parseRepoFile(self):
        try:
            dom = xml.dom.minidom.parseString(self.repoFileText)
        except xml.parsers.expat.ExpatError:
            print "Error while parsing repo file"
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("Invalid repository file"),
                    "nextEnabled" : False,
                    "finishEnabled" : True,
                    "text": _("<p>The file with installation information has invalid format. Please check if it was downloaded properly and run the installer again.</p>")
                    }
                )
            return False
        self.repoFileDom = dom
        return True
    
    def parseErrorMessage(self, message):
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "simpleMessage",
                "pageTitle" : _("Invalid repository file"),
                "nextEnabled" : True,
                "finishEnabled" : False,
                "text": message
                }
            )
        command = self.frontend.waitForInput()
    
    def processRepoFile(self):
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "actionProgress",
                "pageTitle": _("Analyzing installation information"),
                "progressLabel" : _("Analyzing installation information")
                }
            )
        self.frontend.performAction("progressInfo", {"text": _("Analyzing")})
        
        rootElement = self.repoFileDom
        if rootElement.documentElement.tagName != "installinfo":
            dlog(1, "Parse error: no installinfo root element")
            self.parseErrorMessage(_("Structure of installation file is invalid"))
            return False
        #find distribution matching ours
        returncode, text = runSimpleCommandWithTextOutput(["lsb_release", "--id", "--short"])
        self.distributionId = text.strip("\n")
        if returncode != 0:
            dlog(1, "Cannot determine distributionId")
            return False
        returncode, text = runSimpleCommandWithTextOutput(["lsb_release", "--release", "--short"])
        self.distributionRelease = text.strip("\n")
        if returncode != 0:
            dlog(1, "Cannot determine distribution release")
            return False
        
        self.distributionElement = None
        for distribution in rootElement.getElementsByTagName("distribution"):
            dlog(3, "Checking distribution tag: %s" % repr(distribution))
            if distribution.getAttribute("distributionid") != self.distributionId:
                dlog(3, "Distribution id (%s) does not match our distribution id (%s)" % (distribution.getAttribute("distributionid"), self.distributionId))
                continue
            if distribution.getAttribute("release") != self.distributionRelease:
                dlog(3, "Distribution release (%s) does not match our distribution release (%s)" % (distribution.getAttribute("distributionid"), self.distributionId))
                continue
            # both match
            self.distributionElement = distribution
            break
        if self.distributionElement == None:
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("No distribution in installation file"),
                    "nextEnabled" : True,
                    "finishEnabled" : False,
                    "text": _("Installation file does not contain information for this distribution. Cannot proceed with installation.")
                    }
                )
            command = self.frontend.waitForInput()
            return False
        if self.processDistributionElement(self.distributionElement) == False:
            return False
        return True
    
    def processDistributionElement(self, distroElement):
        self.channelElement = None
        for channel in distroElement.getElementsByTagName("channel"):
            if channel.getAttribute("type") == "apt":
                self.channelElement = channel
                break
        if self.channelElement == None:
            self.frontend.performAction(
                "setNextPage", 
                { 
                    "pageName": "simpleMessage",
                    "pageTitle" : _("No recognized channel in installation file"),
                    "nextEnabled" : True,
                    "finishEnabled" : False,
                    "text": _("Installation file does not contain information for handled channel types. Cannot proceed with installation.")
                    }
                )
            command = self.frontend.waitForInput()
            return False
        #now process channel
        reposElements = self.channelElement.getElementsByTagName("repositories")
        if len(reposElements) > 1:
            dlog(1, "Parse error: too many repositories element, count=%s" % len(reposElements))
            self.parseErrorMessage(_("Structure of installation file is invalid"))
            return False
        self.repositories = []
        if len(reposElements) == 0:
            dlog(10, "No repositories defined in installation file")
        else:
            repositoriesElement = reposElements[0]
            repositoryIndex = 0
            for repository in repositoriesElement.getElementsByTagName("repository"):
                repositoryIndex += 1
                repoInfo = dict()
                nameElement = self.findElementForLanguage(repository, "name")
                if nameElement == None:
                    dlog(1, "Parse error: no name element, repository number %s" % repositoryIndex)
                    self.parseErrorMessage(_("Structure of installation file is invalid"))
                    return False
                repoInfo["name"] = getText(nameElement.childNodes)
                keyElements = repository.getElementsByTagName("key")
                if len(keyElements) > 1:
                    dlog(1, "Parse error: too many key elements, count=%s" % len(keyElements))
                    self.parseErrorMessage(_("Structure of installation file is invalid"))
                    return False
                if len(keyElements) > 0:
                    keyElement = keyElements[0]
                    repoInfo["key"] = getText(keyElement.childNodes)
                repositoryData = []
                for repositorydata in repository.getElementsByTagName("repositorydata"):
                    repositoryData.append({"type": repositorydata.getAttribute("type"), "data": getText(repositorydata.childNodes)})
                if len(repositoryData) == 0:
                    dlog(1, "Parse error: no repositorydata elements")
                    self.parseErrorMessage(_("Structure of installation file is invalid"))
                    return False
                repoInfo["repositories"] = repositoryData
                self.repositories.append(repoInfo)
                
            print "Parsed repositories=%s" % repr(self.repositories)
        #now process installInfo
        self.installationSets = []
        
        installationElements = self.channelElement.getElementsByTagName("installation")
        if len(installationElements) == 0:
            dlog(1, "No installation info")
            return True
        
        if len(installationElements) > 1:
            dlog(1, "Parse error: too many installation elements, count=%s" % len(installationElements))
            self.parseErrorMessage(_("Structure of installation file is invalid"))
            return False
        installationElement = installationElements[0]
        
        #now process installation sets
        installationSetIndex = 0
        
        for installSet in installationElement.getElementsByTagName("installationset"):
            installationSetIndex += 1
            installSetInfo = dict()
            installSetInfo["type"] = installSet.getAttribute("type")
            installSetInfo["kind"] = installSet.getAttribute("kind")
            nameElement = self.findElementForLanguage(installSet, "name")
            if nameElement == None:
                dlog(1, "Parse error: no name element, installation set number %s" % installationSetIndex)
                self.parseErrorMessage(_("Structure of installation file is invalid"))
                return False
            installSetInfo["name"] = getText(nameElement.childNodes)
            
            descriptionElement = self.findElementForLanguage(installSet, "description")
            if descriptionElement == None:
                dlog(1, "Parse error: no description element, installation set number %s" % installationSetIndex)
                self.parseErrorMessage(_("Structure of installation file is invalid"))
                return False
            installSetInfo["description"] = getText(descriptionElement.childNodes)
            
            installDataElements = installSet.getElementsByTagName("installdata")
            if len(installDataElements) != 1:
                dlog(1, "Parse error: too many or missing installdata elements, count=%s" % len(installDataElements))
                self.parseErrorMessage(_("Structure of installation file is invalid"))
                return False
            
            installerOptionsElements = installSet.getElementsByTagName("installeroptions")
            if len(installerOptionsElements) > 1:
                dlog(1, "Parse error: too many installeroptions elements, count=%s" % len(installerOptionsElements))
                self.parseErrorMessage(_("Structure of installation file is invalid"))
                return False
            if len(installerOptionsElements) == 1:
                installSetInfo["installeroptions"] = getText(installerOptionsElements[0].childNodes)
            
            installSetInfo["installdata"] = getText(installDataElements[0].childNodes)
            self.installationSets.append(installSetInfo)
            
        print "Parsed installation sets:\n%s" % repr(self.installationSets)
        
        return True
    
    def findElementForLanguage(self, parentElement, tagName):
        supportedLanguages = getLanguagesNames()
        foundElement = None
        foundElementPriority = 1000
        for element in parentElement.getElementsByTagName(tagName):
            language = element.getAttribute("lang")
            if language == "":
                #default element for all languages
                if foundElement == None:
                    foundElement = element
                continue
            if language in supportedLanguages:
                if supportedLanguages.index(language) < foundElementPriority:
                    foundElement = element
                    foundElementPriority = supportedLanguages.index(language)
        return foundElement
                
    def parseRepositoriesFiles(self):
        files = glob.glob("/etc/apt/sources.list.d/*.list")
        files.insert(0, "/etc/apt/sources.list")
        repositories = []

        for line in fileinput.FileInput(files):
            repositories.extend(self.parseRepositoryLine(line))
        return repositories
            
    def parseRepositoryLine(self, line):
        repositories = []
        dlog(10, "Parsing line: %s" % line)
        line = line.strip()
        commentRegex = re.compile("^\s*#")
        if commentRegex.match(line):
            dlog(10, "Line is comment")
            return repositories
        fields = line.split(" ")
        if len(fields) == 0:
            dlog(10, "Line is empty")
            return repositories
        if len(fields) < 4:
            dlog(10, "Not enough words in line")
            return repositories
        for i in xrange(3, len(fields)):
            repository = dict()
            repository["type"] = fields[0]
            repository["url"] = fields[1]
            repository["distro"] = fields[2]
            repository["section"] = fields[i]
            repositories.append(repository)
        return repositories
        
    def installRepositories(self):
        #first check for duplicates in repositories lists
        fileRepositories = self.parseRepositoriesFiles()
        dlog(10, "Parsed repositories files:\n%s" % pprint.pprint(fileRepositories))
        repoLinesToAdd = []
        reposToAdd = []
        self.keysToAdd = []
        for repository in self.repositories:
            #FIXME: choose mirror on some network metric
            chosenMirror = None
            for repoMirror in repository["repositories"]:
                if repoMirror["type"] == "main":
                    chosenMirror = repoMirror
                    break
                #choose any if no main repository
                chosenMirror = repoMirror
            aptLine = chosenMirror["data"]
            parsedRepositories = self.parseRepositoryLine(aptLine)
            addRepo = False
            for repo in parsedRepositories:
                if repo not in fileRepositories:
                    addRepo = True
                    repoLinesToAdd.append("%s %s %s %s" % (repo["type"], repo["url"], repo["distro"], repo["section"]))
            if addRepo == True:
                reposToAdd.append(repository)
                if repository.has_key("key"):
                    self.keysToAdd.append(repository["key"])
        self.repositoriesAdded = False
        if len(reposToAdd) == 0:
            dlog(1, "All repositories are already present")
            return True
        self.repositoriesAdded = True
        reposMessage = "<ul>"
        for repo in reposToAdd:
            reposMessage += "<li>%s</li>" % repo["name"]
        reposMessage += "</ul>"
        
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "simpleMessage",
                "pageTitle" : _("Adding repositories"),
                "nextEnabled" : True,
                "finishEnabled" : False,
                "text": _("<p><u>The following repositories will be now added:</u></p><p>%s</p>") % reposMessage
                }
            )
        command = self.frontend.waitForInput()
        textToAppend = "\n#The following lines were added by One Click Installer:\n"
        textToAppend += "\n".join(repoLinesToAdd) + "\n"
        dlog(10, "Adding the following lines to apt sources list:\n%s" % textToAppend)
        aptSourcesFile = "/etc/apt/sources.list"
        makeFileBackup(aptSourcesFile)
        f = open(aptSourcesFile,"a")
        try:
            f.write(textToAppend)
        finally:
            f.close()
    
    def installKeys(self):
        #add keys
        returncode, text = runSimpleCommandWithTextOutput(["/usr/bin/apt-key", "list"])
        if returncode != 0:
            dlog(1, "Cannot run apt-list, error=%s" % returncode)
            return False
        
        currentKeys = self.parseGpgKeyOutput(text)
        
        keysToAdd = []
        for key in self.keysToAdd:
            returncode, text = runSimpleCommandWithTextOutputAndInput(["/usr/bin/gpg", "-"], key)
            if returncode != 0:
                dlog(1, "Cannot parse key, error=%s" % returncode)
                self.parseErrorMessage("Key specified in repository file is not valid")
                return False
            keysInFile = self.parseGpgKeyOutput(text)
            for keyid in keysInFile.keys():
                if currentKeys.has_key(keyid):
                    dlog(1, "Key %s already present" % keyid)
                    continue
                dlog(1, "Key %s will be added" % keyid)
                keyData = keysInFile[keyid]
                keyData["keyid"] = keyid
                keyData["data"] = key
                keysToAdd.append(keyData)
        if len(keysToAdd) == 0:
            dlog(1, "All necessary keys are already installed")
            return
        message = "<ul>"
        for key in keysToAdd:
            message += "<li>%s [%s]</li>" % (key["owner"], key["keyid"])
        message += "</ul>"
        
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "simpleMessage",
                "pageTitle" : _("Adding keys"),
                "nextEnabled" : True,
                "finishEnabled" : False,
                "text": _("<p><u>The following keys will be now added:</u></p><p>%s</p>") % message
                }
            )
        command = self.frontend.waitForInput()
        for key in keysToAdd:
            returncode, text = runSimpleCommandWithTextOutputAndInput(["apt-key", "add", "-"], key["data"])
            if returncode != 0:
                dlog(1, "Cannot add key, error=%s" % returncode)
                self.parseErrorMessage("Failed to add key %s [%s]" % (key["owner"], key["keyid"]))
                return False
        
        return True
    def parseGpgKeyOutput(self, output):
        #pub   1024D/81836EBF 2006-06-26
        regexPub = re.compile(r"^\s*pub\s+\w+/(\w+)\b\s+(.*)$")
        regexUid = re.compile(r"^\s*uid\s+(.*)$")
        keys = dict()
        keyid = None
        mode = None

        for line in output.split("\n"):
            dlog(10, "Parsing key line: %s" % line)
            if mode == "uid":
                matchObj = regexUid.match(line)
                if matchObj != None:
                    dlog(10, "Matching line: %s" % line)
                    keyOwner = matchObj.group(1)
                keys[keyid] = {"owner": keyOwner}
                mode = None
            #no else - on purpose, to allow fallback
            if mode == None:
                matchObj = regexPub.match(line)
                if matchObj == None:
                    continue
                dlog(10, "Matching line: %s" % line)
                keyid = matchObj.group(1)
                keyOwner = matchObj.group(2)
                dlog(10, "keyid=%s" % keyid)
                dlog(10, "matches=%s" % repr(matchObj.groups()))
                mode = "uid"
            else:
                keyid = None
        dlog(10, "Parsed keys:\n%s" % repr(keys))
        return keys
    def installSets(self):
        installationChoices = []
        if len(self.installationSets) == 0:
            dlog(10, "No installation sets present, skipping installation step")
            return True
        
        for installationSet in self.installationSets:
            enabled = True
            checked = False
            if installationSet["type"] == "required":
                checked = True
                enabled = False
            elif installationSet["type"] == "recommended":
                checked = True
            else:
                checked = False
            installationChoices.append({
                "id": installationSet["installdata"],
                "text": installationSet["name"], 
                "description" : "<p><i>" + installationSet["description"] + "</i></p>",
                "enabled": enabled,
                "checked" : checked
                })
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "simpleMultichoice",
                "pageTitle" : _("Select packages to install"),
                "text": _("The following applications are available for installation. You can select whether to install optional packages, required packages cannot be unchecked."),
                "choices": installationChoices
                }
            )
        restartOutput = self.frontend.waitForInput()
        
        selectedChoices = restartOutput["choices"]
        dlog(10, "User selected: %s" % repr(selectedChoices))
        
        #ugly hack with options for Skype which does not sign its packages :-/
        for installationSet in self.installationSets:
            if installationSet.has_key("installeroptions") and installationSet["installdata"] in selectedChoices:
                dlog(1, "Installing %s separately with options \"%s\"" % (installationSet["installdata"], installationSet["installeroptions"]))
                if self.installPackageSets([installationSet["installdata"], ], installationSet["installeroptions"].split()) == False:
                    return False
                selectedChoices.remove(installationSet["installdata"])
                
                
        if len(selectedChoices) > 0:
            if self.installPackageSets(selectedChoices) == False:
                return False
        return True
    def installPackageSets(self, packageSets, aptOptions = None):
        self.frontend.performAction(
            "setNextPage", 
            {
                "pageName": "actionProgress",
                "pageTitle": _("Installation of software"),
                "progressLabel" : _("Installing")
                }
            )
            
        actionWrapper = LongActionWrapper(
            self.installPackageLineOutput,
            self.installPackageHeartbeat
            )
            
        if True: #self.repositoriesAdded:
            self.frontend.performAction("progressInfo", {"text": _("Updating repositories...")})
            
            command = ["apt-get", "update"]
            
            returncode = actionWrapper.runCommand(
                command,
                env = englishEnviron()
                )
            
            if returncode != 0:
                dlog(1, "Updating repositories failed with error %d" % returncode)
                return False
            dlog(1, "Updating repositories finished successfully" )
        packages = []
        for packageSet in packageSets:
            packages.extend(packageSet.split())
        if self.installPackages(packages, aptOptions) == False:
            return False
        return True
    
    def doStart(self, params):
        firstLoop = True
        while True:
            if firstLoop == True:
                firstLoop = False
            else:
                self.frontend.performAction(
                    "setNextPage", 
                    {
                        "pageName": "simpleChoice",
                        "pageTitle" : _("Wizard failure"),
                        "text": _("One of the operations failed.\nDo you want to restart the wizard from the beginning or quit the wizard?"),
                        "choices": {
                            "restart": {"text": _("Restart wizard"), "checked" : True },
                            "quit": {"text": _("Quit wizard") }
                            }
                        }
                    )
                restartOutput = self.frontend.waitForInput()
                
                if restartOutput["choice"] == "quit":
                    self.frontend.performAction(
                        "setNextPage", 
                        {
                            "pageName": "simpleMessage",
                            "pageTitle" : _("Wizard failure"),
                            "nextEnabled" : False,
                            "finishEnabled" : True,
                            "text": _("<p>Thank you for using One click installer.</p><p>If you think the behaviour spotted is due to a bug in application, please report bug to the author, along with output from console.</p><p>Click \"Finish\" to quit wizard.</p>"),
                            }
                        )
                    return
                else:
                    pass #Move to first wizard step
            if os.path.exists(self.repoFileName) == False:
                self.frontend.performAction(
                    "setNextPage", 
                    {
                        "pageName": "simpleMessage",
                        "pageTitle" : _("File does not exist"),
                        "nextEnabled" : False,
                        "finishEnabled" : True,
                        "text": _("<p>The file with installation information does not exist. Wizard will now quit.</p>"),
                        }
                    )
                return
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("Welcome"),
                    "nextEnabled" : True,
                    "finishEnabled" : False,
                    "text": _("<p>Welcome to one click installer. This wizard will install requested software.</p>"),
                    }
                )
            command = self.frontend.waitForInput()

            root, ext = os.path.splitext(self.repoFileName)
            if ext == ".deb":
                if self.installDebPackage(self.repoFileName) == False:
                    continue
            else:
                #install from oci file
                if self.parseAndVerifyRepoFile() == False:
                    continue
                
                if self.processRepoFile() == False:
                    continue
                
                if self.installRepositories() == False:
                    continue
                
                if self.installKeys() == False:
                    continue
                
                if self.installSets() == False:
                    continue
            #continue common for deb and oci
            self.frontend.performAction(
                "setNextPage", 
                {
                    "pageName": "simpleMessage",
                    "pageTitle" : _("Success"),
                    "nextEnabled" : False,
                    "finishEnabled" : True,
                    "text": _("<p>Installation has been successful.</p>"),
                    }
                )
            return
            
            
        print "Exiting backend thread"
